{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Week 8 特征工程 作业三"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 输入数据，数据准备与处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 导入数据函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#--------------------------\n",
    "# 导入数据函数\n",
    "#--------------------------\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "def load_housing_data(housing_path = './'):\n",
    "    csv_path = os.path.join(housing_path, 'housing.csv')\n",
    "    return pd.read_csv(csv_path)\n",
    "\n",
    "housing = load_housing_data()\n",
    "\n",
    "# 按照 income一列的数值分层（5类）\n",
    "# pd.cut()的作用，是把连续值转换成类别标签 ，加入housing中新建一列\n",
    "housing['income_cat'] = pd.cut(housing['median_income'],bins = [0., 1.5, 3.0, 4.5, 6., np.inf], labels = [1,2,3,4,5])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 分层分割训练集和测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#--------------------------\n",
    "# 按照 income 一列的数值分层分割训练集和测试集（80%：20%）\n",
    "#--------------------------\n",
    "from sklearn.model_selection import StratifiedShuffleSplit  \n",
    "\n",
    "# 函数先将样本随机打乱，然后根据设置参数划分出指定数量的独立的train/test数据集。\n",
    "split  = StratifiedShuffleSplit(n_splits = 1, test_size = 0.2, random_state  = 42)\n",
    "\n",
    "for train_index, test_index in split.split(housing, housing['income_cat']):\n",
    "    strat_train_set = housing.loc[train_index]  # 训练集\n",
    "    strat_test_set = housing.loc[test_index]    # 测试集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 机器学习算法的数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#--------------------------    \n",
    "# 机器学习算法的数据准备  results： housing_labels  vs  housing_final \n",
    "#--------------------------\n",
    "# 标签列\n",
    "housing_labels = strat_train_set['median_house_value'].copy()\n",
    "testIndex = housing_labels.index\n",
    "\n",
    "# 数据列\n",
    "housing1 = strat_train_set.drop(['median_house_value', 'income_cat'], axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 自定义转换器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 转换器\n",
    "from sklearn.base import BaseEstimator, TransformerMixin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 根据列名选择DataFrame中的列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    # 根据列名选择DataFrame中的列\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names].values  \n",
    "        #return pd.DataFrame(X[self.attribute_names].values, index = X.index, columns = self.attribute_names)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 增加组合列 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "rooms_ix, bedrooms_ix, population_ix, household_ix = [list(housing1.columns).index(col) for col in ('total_rooms','total_bedrooms','population','households')]\n",
    "\n",
    "# 构造的附加组合列列名\n",
    "extra_attribs = []\n",
    "\n",
    "class CombinedAttributesAdder(BaseEstimator, TransformerMixin):\n",
    "    # 增加附加组合列 \n",
    "    def __init__(self, add_bedrooms_per_room = True):\n",
    "        self.add_bedrooms_per_room = add_bedrooms_per_room\n",
    "        \n",
    "    def fit(self, X, y = None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X, y = None):\n",
    "        rooms_per_household = X[:,rooms_ix] / X[:,household_ix]\n",
    "        population_per_household = X[:,population_ix] / X[:,household_ix]\n",
    "        global extra_attribs\n",
    "        \n",
    "        if self.add_bedrooms_per_room:           \n",
    "            extra_attribs = [\"rooms_per_household\", \"population_per_household\", \"bedrooms_per_room\"]            \n",
    "            bedrooms_per_room = X[:,bedrooms_ix]/X[:,rooms_ix]\n",
    "            # np.c_()函数将多个DataFrame，Series对象合并成一个\n",
    "            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]\n",
    "        else:\n",
    "            extra_attribs = [\"rooms_per_household\", \"population_per_household\"]\n",
    "            return np.c_[X, rooms_per_household, population_per_household]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 将文本数据列转换成独热编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_one_hot_attribs = []\n",
    "class MyLabelBinarizer(TransformerMixin):\n",
    "    # 将文本数据列转换成独热编码\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        self.encoder = LabelBinarizer(*args, **kwargs)\n",
    "    def fit(self, x, y=0):\n",
    "        self.encoder.fit(x)\n",
    "        return self\n",
    "    def transform(self, x, y=0):\n",
    "        global cat_one_hot_attribs \n",
    "        cat_one_hot_attribs = list(self.encoder.classes_) \n",
    "        return self.encoder.transform(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 转换流水线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# 转换流水线\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "# 数值列流水线\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "# 对数值列作处理的Pipeline\n",
    "housing_num = housing1.drop('ocean_proximity', axis = 1)\n",
    "num_attribs = list(housing_num) \n",
    "\n",
    "add_bedrooms_per_room = True\n",
    "num_pipeline = Pipeline([\n",
    "        #名称/转换器\n",
    "        ('selector', DataFrameSelector(num_attribs)),\n",
    "        ('imputer', SimpleImputer(strategy=\"median\")),\n",
    "        ('attribs_adder', CombinedAttributesAdder(add_bedrooms_per_room)),\n",
    "        ('std_scaler', StandardScaler()),\n",
    "    ])\n",
    "\n",
    "# 文字列流水线\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "# 对文字列作处理的Pipeline\n",
    "cat_attribs = ['ocean_proximity']\n",
    "#cat_one_hot_attribs = list(encoder.classes_)\n",
    "\n",
    "cat_pipeline = Pipeline([\n",
    "        ('selector', DataFrameSelector(cat_attribs)),               \n",
    "        ('LabelBinarizer', MyLabelBinarizer()),\n",
    "    ])\n",
    "\n",
    "# 返回特征列名\n",
    "attributes = []\n",
    "def AttriNames():    \n",
    "    global attributes, num_attribs, extra_attribs, cat_one_hot_attribs\n",
    "        # 将上述所有列名组合成一个列名列表（最终数据的每列列名）\n",
    "    attributes = num_attribs + extra_attribs + cat_one_hot_attribs\n",
    "    return attributes\n",
    "\n",
    "# 构造DataFrame\n",
    "class DFConstruct(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, index = None, columns = None):\n",
    "        self.index = index\n",
    "        self.columns = columns        \n",
    "    def fit(self, X_df, y = None):\n",
    "        return self    \n",
    "    def transform(self, X_df, y = None):\n",
    "        if self.columns == None:\n",
    "            self.columns = AttriNames()\n",
    "            print(X_df.shape)\n",
    "        return pd.DataFrame(X_df, index = self.index, columns = self.columns)  \n",
    "\n",
    "# 组合多个流水线\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "full_pipeline = FeatureUnion(transformer_list=[\n",
    "        ('num_pipline', num_pipeline,),\n",
    "        ('cat_pipline', cat_pipeline),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* 根据皮尔逊相关系数挑选重要特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 选出特征相关性最大的k项特征 \n",
    "important_attr_name = []\n",
    "\n",
    "# 返回前k项元素在原数组中的index位置\n",
    "def indices_of_top_k(arr, k):\n",
    "    #return np.sort(np.argpartition(np.array(arr), -k)[-k:])\n",
    "    return np.argsort(np.array(arr))[-1:-k-1:-1]\n",
    "\n",
    "class ImportantAttrSelector(BaseEstimator, TransformerMixin):\n",
    "    # 根据皮尔逊相关系数挑选重要特征\n",
    "    def __init__(self, k, y):\n",
    "        self.k = k\n",
    "        #self.columns = featurelist\n",
    "        self.y = y\n",
    "        \n",
    "    def fit(self, X_df, y = None):\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X_df, y = None):\n",
    "        k = self.k\n",
    "        global corr_matrix\n",
    "        corr_matrix = pd.concat([X_df, self.y], axis = 1).corr()  # 计算相关系数\n",
    "        corr_ser = corr_matrix.iloc[:-1,-1] # 选取与y相关一列，并忽略自相关一项\n",
    "        \n",
    "        global important_attr_all, important_attr, important_attr_name  \n",
    "        print(corr_ser)\n",
    "        #print(self.columns)\n",
    "        \n",
    "        corr_ser2 = pd.concat([np.abs(corr_ser), corr_ser], axis = 1)\n",
    "        \n",
    "        corr_ser2.columns = ['abs', 'corr']\n",
    "        \n",
    "        important_attr_all = corr_ser2.sort_values(by = 'abs', axis = 0, ascending = False)\n",
    "        important_attr = important_attr_all['corr'][0:k]\n",
    "        important_attr_name = important_attr_all.index[0:k]\n",
    "        print(important_attr_name)\n",
    "        return X_df[important_attr_name].values\n",
    "\n",
    "# 选取重要特征流水线\n",
    "#ImportantSelector_pipline = Pipeline([\n",
    "#        ('ImportantSelector', ImportantAttrSelector(k = 8,featurelist = attributes, y = housing_labels)),\n",
    "#])\n",
    "\n",
    "#housing_pl = ImportantSelector_pipline.fit_transform(housing_tr)\n",
    "#housing_final = pd.DataFrame(housing_pl, index = housing_tr.index, columns = important_attr_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 选择和训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 使用随机森林模型训练、交叉验证、及网格搜索调参\n",
    "from sklearn.ensemble import RandomForestRegressor  # 随机森林模型\n",
    "from sklearn.model_selection import cross_val_score  # 交叉验证 \n",
    "from sklearn.model_selection import GridSearchCV  # 网格搜索调参\n",
    "\n",
    "# 网格搜索参数\n",
    "param_grid = [\n",
    "    {'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},\n",
    "    {'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]},\n",
    "  ]\n",
    "\n",
    "class MyRandomForestRegressorCV(BaseEstimator, TransformerMixin):\n",
    "    # 根据列名选择DataFrame中的列\n",
    "    def __init__(self, y):\n",
    "        self.y = y\n",
    "        \n",
    "    def fit(self, X_df, y = None):\n",
    "        forest_reg = RandomForestRegressor(random_state=42)        \n",
    "        grid_search = GridSearchCV(forest_reg, param_grid, cv=5,\n",
    "                           scoring='neg_mean_squared_error', return_train_score=True)\n",
    "        \n",
    "        grid_search.fit(X_df, self.y)\n",
    "        self.model = grid_search\n",
    "        return self\n",
    "    \n",
    "    def transform(self, X_df):\n",
    "        return X_df, self.model   \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 完整流水线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据预处理，选择重要特征、优化模型、预测数据\n",
    "prepare_select_and_predict_pipeline = Pipeline([\n",
    "        ('preparation', full_pipeline),\n",
    "        #('AttrName', AttriNames()),\n",
    "        ('ConstructDF', DFConstruct(testIndex)),\n",
    "        ('ImportantSelector', ImportantAttrSelector(k = 7, y = housing_labels)),\n",
    "        ('fitting', MyRandomForestRegressorCV(housing_labels)),\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(16512, 16)\n",
      "longitude                  -0.047432\n",
      "latitude                   -0.142724\n",
      "housing_median_age          0.114110\n",
      "total_rooms                 0.135097\n",
      "total_bedrooms              0.047642\n",
      "population                 -0.026920\n",
      "households                  0.064506\n",
      "median_income               0.687160\n",
      "rooms_per_household         0.146285\n",
      "population_per_household   -0.021985\n",
      "bedrooms_per_room          -0.234240\n",
      "<1H OCEAN                   0.259521\n",
      "INLAND                     -0.482886\n",
      "ISLAND                      0.013709\n",
      "NEAR BAY                    0.158733\n",
      "NEAR OCEAN                  0.137378\n",
      "Name: median_house_value, dtype: float64\n",
      "Index(['median_income', 'INLAND', '<1H OCEAN', 'bedrooms_per_room', 'NEAR BAY',\n",
      "       'rooms_per_household', 'latitude'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n",
      "c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py:552: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details: \n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 531, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 392, in fit\n",
      "    for i, t in enumerate(trees))\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 1029, in __call__\n",
      "    if self.dispatch_one_batch(iterator):\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 847, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 765, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 208, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 572, in __init__\n",
      "    self.results = batch()\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\joblib\\parallel.py\", line 253, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\ensemble\\_forest.py\", line 168, in _parallel_build_trees\n",
      "    tree.fit(X, y, sample_weight=curr_sample_weight, check_input=False)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 1246, in fit\n",
      "    X_idx_sorted=X_idx_sorted)\n",
      "  File \"c:\\python\\python37\\lib\\site-packages\\sklearn\\tree\\_classes.py\", line 279, in fit\n",
      "    raise ValueError(\"max_features must be in (0, n_features]\")\n",
      "ValueError: max_features must be in (0, n_features]\n",
      "\n",
      "  FitFailedWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>median_income</th>\n",
       "      <th>INLAND</th>\n",
       "      <th>&lt;1H OCEAN</th>\n",
       "      <th>bedrooms_per_room</th>\n",
       "      <th>NEAR BAY</th>\n",
       "      <th>rooms_per_household</th>\n",
       "      <th>latitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17606</th>\n",
       "      <td>-0.614937</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.155318</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.312055</td>\n",
       "      <td>0.771950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18632</th>\n",
       "      <td>1.336459</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.836289</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.217683</td>\n",
       "      <td>0.659695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14650</th>\n",
       "      <td>-0.532046</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.422200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.465315</td>\n",
       "      <td>-1.342183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3230</th>\n",
       "      <td>-1.045566</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.196453</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.079661</td>\n",
       "      <td>0.313576</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3555</th>\n",
       "      <td>-0.441437</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.269928</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.357834</td>\n",
       "      <td>-0.659299</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       median_income  INLAND  <1H OCEAN  bedrooms_per_room  NEAR BAY  \\\n",
       "17606      -0.614937     0.0        1.0           0.155318       0.0   \n",
       "18632       1.336459     0.0        1.0          -0.836289       0.0   \n",
       "14650      -0.532046     0.0        0.0           0.422200       0.0   \n",
       "3230       -1.045566     1.0        0.0          -0.196453       0.0   \n",
       "3555       -0.441437     0.0        1.0           0.269928       0.0   \n",
       "\n",
       "       rooms_per_household  latitude  \n",
       "17606            -0.312055  0.771950  \n",
       "18632             0.217683  0.659695  \n",
       "14650            -0.465315 -1.342183  \n",
       "3230             -0.079661  0.313576  \n",
       "3555             -0.357834 -0.659299  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "housing_pl,estimator = prepare_select_and_predict_pipeline.fit_transform(housing1)\n",
    "#housing_pl = prepare_select_and_predict_pipeline.fit_transform(housing1)\n",
    "# 将上述所有列名组合成一个列名列表（最终数据的每列列名）\n",
    "#attributes = num_attribs + extra_attribs + cat_one_hot_attribs\n",
    "\n",
    "housing_tr = pd.DataFrame(housing_pl, index = housing1.index, columns = important_attr_name)\n",
    "housing_tr.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_features': 2, 'n_estimators': 30}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "estimator.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预测"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 判断增加bedrooms_per_room这个合成特征，是否对预测精度有好处"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 2 candidates, totalling 10 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=4)]: Done  10 out of  10 | elapsed:    0.2s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(16512, 16)\n",
      "longitude                  -0.047432\n",
      "latitude                   -0.142724\n",
      "housing_median_age          0.114110\n",
      "total_rooms                 0.135097\n",
      "total_bedrooms              0.047642\n",
      "population                 -0.026920\n",
      "households                  0.064506\n",
      "median_income               0.687160\n",
      "rooms_per_household         0.146285\n",
      "population_per_household   -0.021985\n",
      "bedrooms_per_room          -0.234240\n",
      "<1H OCEAN                   0.259521\n",
      "INLAND                     -0.482886\n",
      "ISLAND                      0.013709\n",
      "NEAR BAY                    0.158733\n",
      "NEAR OCEAN                  0.137378\n",
      "Name: median_house_value, dtype: float64\n",
      "Index(['median_income', 'INLAND', '<1H OCEAN', 'bedrooms_per_room', 'NEAR BAY',\n",
      "       'rooms_per_household', 'latitude'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'preparation__num_pipline__attribs_adder__add_bedrooms_per_room': True}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RandomForestRegressor\n",
    "\n",
    "# 用最优模型重新定义流水线\n",
    "prepare_select_and_predict_pipeline2 = Pipeline([\n",
    "        ('preparation', full_pipeline),\n",
    "        ('ConstructDF', DFConstruct(testIndex)),\n",
    "        ('ImportantSelector', ImportantAttrSelector(k = 7, y = housing_labels)),\n",
    "        ('fitting', RandomForestRegressor(random_state=42, max_features = 2, n_estimators = 30)),\n",
    "    ])\n",
    "\n",
    "# 网格搜索参数\n",
    "param_grid_2 = [{\n",
    "      'preparation__num_pipline__attribs_adder__add_bedrooms_per_room':[True, False]\n",
    "}]\n",
    "\n",
    "grid_search_prep = GridSearchCV(prepare_select_and_predict_pipeline2, param_grid_2, cv=5,\n",
    "                                scoring='neg_mean_squared_error', verbose=2, n_jobs=4)\n",
    "grid_search_prep.fit(housing1, housing_labels)\n",
    "grid_search_prep.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
